package net.bwie.realtime.jtp.dwd.log.job;
import com.alibaba.fastjson.JSON;


import net.bwie.realtime.jtp.common.utils.KafkaUtil;

import net.bwie.realtime.jtp.dwd.function.AdjustIsNewProcessFunction;
import net.bwie.realtime.jtp.dwd.function.LogSplitProcessFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

public class JtpLogEtIJob {
    /*
    *   实时日志ETL
    *  1.从kafka中读取数据
    *
    * */
    public static void main(String[] args)throws Exception {
        //1.创建执行环境
        StreamExecutionEnvironment  env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        // 启用 Checkpointing（例如每5秒一次）
//        env.enableCheckpointing(5000);
        //2.从kafka中读取数据
        DataStream<String> dataStream = KafkaUtil.consumerKafka(env, "topic-log");
        //3.打印数据

        dataStream.print();
        prcessLog(dataStream);

        env.execute("JtpLogEtIJob");

    }
    private static void prcessLog(DataStream<String> dataStream) {
        //1.数据清洗
        DataStream<String> jsonStream = logCleaned(dataStream);
        //2.新老访客状态标记修复
        DataStream<String> etlStream = prcessIsNew(jsonStream);
        //3.数据分拆
        DataStream<String> pageStream = splitStream(etlStream);
        pageStream.print("page");
        //4.数据写入kafka
        KafkaUtil.producerKafka(pageStream,"dwd-traffic-page-log");
    }

    private static DataStream<String> splitStream(DataStream<String> etlStream) {
        //创建侧输出流
        final OutputTag<String> errorTag = new OutputTag<String>("error-log") {};
        final OutputTag<String> startTag = new OutputTag<String>("start-log") {};
        final OutputTag<String> displayTag = new OutputTag<String>("display-log") {};
        final OutputTag<String> actionTag = new OutputTag<String>("action-log") {};
        SingleOutputStreamOperator<String> pageStream = etlStream.process(
                new LogSplitProcessFunction(errorTag,startTag,displayTag,actionTag)
        );

        DataStream<String> errorStream = pageStream.getSideOutput(errorTag);
        KafkaUtil.producerKafka(errorStream,"dwd-traffic-error-log");
        DataStream<String> startStream = pageStream.getSideOutput(startTag);
        KafkaUtil.producerKafka(startStream,"dwd-traffic-start-log");
        DataStream<String> displayStream = pageStream.getSideOutput(displayTag);
        KafkaUtil.producerKafka(displayStream,"dwd-traffic-display-log");
        DataStream<String> actionStream = pageStream.getSideOutput(actionTag);
        KafkaUtil.producerKafka(actionStream,"dwd-traffic-action-log");
        return pageStream;

    }

    private static DataStream<String> prcessIsNew(DataStream<String> jsonStream) {
        KeyedStream<String, String> midStream = jsonStream.keyBy(
                new KeySelector<String, String>() {
                    @Override
                    public String getKey(String value) throws Exception {
                        return JSON.parseObject(value).getJSONObject("common").getString("mid");
                    }
                }
        );
        SingleOutputStreamOperator<String> isNewStream = midStream.process(new AdjustIsNewProcessFunction());
        return isNewStream;
    }


    private static DataStream<String> logCleaned(DataStream<String> logStream) {
        //脏数据侧边流输出时标记
        OutputTag<String> dirtyTAg = new OutputTag<String>("dirty-log") {
        };
        //脏数据清洗处理
        SingleOutputStreamOperator<String> cleanedStream = logStream.process(new ProcessFunction<String, String>() {
            @Override
            public void processElement(String value, Context ctx, Collector<String> out) throws Exception {
                try {
                    //1.解析json数据
                    JSON.parseObject(value);
                    out.collect(value);
                } catch (Exception e) {
                    ctx.output(dirtyTAg, value);
                }
            }
        });
        //脏数据输出到kafka
        DataStream<String> dirtyStream = cleanedStream.getSideOutput(dirtyTAg);
        KafkaUtil.producerKafka(dirtyStream, "dwd-traffic-dirty-log");
        //返回清洗后的数据
        return cleanedStream;
    }
}
